  ' Build Command List v 2.4 - KEYWORDS.BAS
  ' Hugh Buckle May 2013 - Updated Dec 2014
  '*****************************************************
  ' This is part of the Crunch suite of programs. It prepares the two
  ' lists of MMBasic reserved words used in the other programs. It only
  ' needs to be run when MMBasic is updated with new reserved words.
  '
  ' Loads all the known reserved words.
  ' Converts them to uppercase.
  ' Sorts them alphabetiacally.
  ' Removes duplicates.
  ' Writes them as a CSV file with each record no more than 254 chars long.
  ' The resulting file is used by CRUNCH to recognise commands.
  '
  ' Also writes a subset of one or two character reserved words.
  ' This latter file is used by CRUNCHa to stop short reserved words
  ' from being used as variables, labels and function/subroutine names.
  '*****************************************************
  ' To execute the program either
  ' - create a parm file RWSRTMRG.DAT containing a list of input files,
  '   separated by a blank, then run RWSRTMRG from the command line or
  ' - enter the command RWSRTMRG <InFile1> [<InFile2>[ <InFile3>...]]
  '*****************************************************
  'Mod 20/7/2013 Added max length of words in ShortRWord file to front of
  '              file so CRUNCHA can set length of Short Word array elements.
  'Mods 1/12/2014
  '            - Allow multiple reserved words per line in input files.
  '            - Remove any blanks from input files.
  '            - Remove input file count from command line (MM.CMDLine$).
  '            - Print a warning if a full stop is found in an input file
  '              except when part of a system variable. e.g. MM.HRES
  '            - Restructured the parameter list. Now just needs a list of
  '              input file names. Output file names are now fixed.

Initialise:
  True=1: False=0
  Testing=false
  FullStopFound=false
  CSVRecLen=254          'Max length of each CSV record
  ShortRWordLen=2        'Max length of words to be written to ShortCMDFile$
  HiASCII$=Chr$(255)     'Highest ascii character
  If MM.Device$="Colour Maximite" Then Mode 1
  If MM.Device$="Maximite" Then
    ArrayMax=230
  ElseIf MM.Device$="DOS" Then
    ArrayMax=200
  EndIf

  ' GetFileNames gets the input and output file names from
  ' ParmFile or MM.CMDLine$, opens them and checks for errors.
  ' On completions
  '  - Output filename is in CSVFile$
  '  - Short Command filename is in ShortCMDFile$
  '  - Input filenames are in Infile$() array

  'Default file names
  ParmFile$     ="KeyWords.DAT"
  CSVFile$      ="RWord.CSV"
  TempShortFile$="RWords.tmp"
  ShortCMDFile$ ="RWords.txt"
  MaxInputFiles=10
  GetFileNames

  ' Now create a temporary file to hold sorted data.
  ' There may need to be more than one so keep a count.


Mainline_Sort_Stage:
  '*****************************************************
  'Read unsorted reserved word data into an array, sort it
  'into ascii sequence and write it to a temp file.
  '*****************************************************
  NoMoreInputData=False
  InputFileNo=1
  Open InFile$(InputFileNo) For Input As #1
  Print "  Reading     ";InFile$(InputFileNo)
  NoTempFiles=0

  Do
    Dim a$(ArrayMax) ' Array to contain each Reserved word ready for sorting
    LoadRWordArray(WordCount,NoMoreInputData)
    TotWordCount=TotWordCount+WordCount

    Print "  Sorting     ";Str$(WordCount);" reserved words"
    Bsort(WordCount)
    Print

    NoTempFiles=NoTempFiles+1
    TempFile$="RWTemp"+Str$(NoTempFiles)+".TMP"
    Print "  Writing temporary sorted reserved words to ";TempFile$
    Open TempFile$ For Output As #2
    WriteTempFile(WordCount)
    Close #2
    Print

    Erase a$    'Clear the Reserved word array
  Loop Until NoMoreInputData

  If FullStopFound Then
    Print
    Print "   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
    Print "  { Check input files for full stops between reserved words and rerun.}"
    Print "  { OUTPUT FILES HAVE NOT BEEN WRITTEN.                               }"
    Print "   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
    Print
    End
  EndIf

Mainline_Merge_Stage:
  '*****************************************************
  'Merge the temporary files into one, dropping any duplicates
  'and write them as a CSV file, filling each record with as many
  'words as will fit in 256 characters.
  'Output CSV file is file #1
  'Temporary input files are File #2 to #n+1 (n=number of temp files)
  '*****************************************************
  Dim TempFile$(NoTempFiles)   'Holds filename of temp files
  Dim a$(NoTempFiles)         'Holds current value read from each temp file

  Print "  Opening output CSV file ";CSVFile$
  Open CSVFile$ For output As #9
  Open TempShortFile$ For output As #10

  Print "  Merging temporary sorted words to ";CSVFile$
  For TempFileNo=1 To NoTempFiles
    TempFile$(TempFileNo)="RWTemp"+Str$(TempFileNo)+".TMP"
    Open TempFile$(TempFileNo) For input As #TempFileNo
    Input #TempFileNo,a$(TempFileNo)    'Read the first word from each file
    If Eof(TempFileNo) Then
      a$(TempFileNo)=HiASCII$
      Print "EOF ";TempFile$(TempFileNo); " at First read."
    EndIf
  Next

  PrevWord$=""

  Do
    b$=HiASCII$
    SelectLowestWord(b$,TempFileNo)
    If b$<>HiASCII$ Then
      If b$<>PrevWord$ Then           'Ignor duplicates
        WriteCSVRecord(b$)
        WriteTempShortCMDFile(b$)
        PrevWord$=b$
      Else
        DupCount=DupCount+1
      EndIf
    EndIf
    If Not Eof(TempFileNo) Then
      Input #TempFileNo,a$(TempFileNo)    'Read the next word from that file
    Else
      a$(TempFileNo)=HiASCII$
    EndIf

  Loop Until b$=HiASCII$    'will be HiASCII when all input files at EOF

  Print #9, CSVRecord$      'Write the last CSV Record
  CSVRecCount=CSVRecCount+1
  Close #9,#10

  'Finally read the temporary file of one and two character reserved
  'words and write its final file, adding the number of records it
  'contains on the front.
  WriteShortCMDFile

  Print "  Removing temporary files."
  Print
  Kill TempShortFile$
  For i=1 To NoTempFiles
    Close #i
    Kill TempFile$(i)
  Next

  'Print record counts for checking
  Print "  ";TotWordCount, "Words read."
  Print "  ";DupCount, "Duplicates found."
  Print "  ";RWordCount, "Words written in ";CSVRecCount;" Records to ";CSVFile$""
  Print "  ";ShortRwords, "Short reserved words written to ";ShortCMDFile$

  'If FullStopFound then
  '  print
  '  print " ` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
  '  Print "  { Check input files for full stops between reserved words and rerun. }"
  '  print "   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
  '  Print
  'endif

End
  '===============================================================
  '===============================================================

Sub LoadRWordArray(Count,NoMoreInputData)
  '*****************************************************
  'Fill the array with reserved words.
  '*****************************************************
  Count=1
  Do
    Line Input #1,a$
    a$=UCase$(a$)

    'Remove any blanks from the input record
    Do
      p=Instr(1,a$," ")
      If p>1 And p<Len(a$) Then
        a$=Left$(a$,p-1)+Mid$(a$,p+1)
      ElseIf p=1 Then
        a$=Mid$(a$,2)
      ElseIf p=Len(a$) Then
        A$=Left$(a$,p-1)
      EndIf
    Loop Until p=0

    'Warn of any full stops which are not part of a system variable
    p=Instr(1,a$,".")
    If p<>0 Then
      If p<3 Then
        PrintFullStopWarning(a$,InFile$(InputFileNo))
      ElseIf P>2 And Mid$(a$,p-2,2)<>"MM") Then
        PrintFullStopWarning(a$,InFile$(InputFileNo))
      EndIf
    EndIf

    ps=1               ' ps and pe are start and end pointers

    Do                 'separate parameters from the command
      pe=Instr(ps,a$,",")
      If pe=0 Then
        a$(Count)=a$
      Else
        a$(Count)=Left$(a$,pe-1)
        a$=Mid$(a$,pe+1)
      EndIf
      Count=Count+1
    Loop Until pe=0

    If Eof(#1) Then
      Close #1
      If InputFileNo<NbrInputFiles Then
        InputFileNo=InputFileNo+1
        Open InFile$(InputFileNo) For Input As #1
        Print "  Reading     ";InFile$(InputFileNo)
      Else
        NoMoreInputData=True
      EndIf
    EndIf
  Loop Until Count>ArrayMax-10 Or NoMoreInputData
End Sub 'LoadRWordArray

Sub PrintFullStopWarning(a$,b$)
  Local k$
  Print
  Print "WARNING: Unexpected full stop found in file ";b$;" in record "
  Print Space$(9);a$
  If FullStopFound=False Then
    Print Space$(9);"Probably should be replaced with a comma."
    Print Space$(9);"Press a key to continue checking."
    Print Space$(9);"Output files will not be written."
  EndIf
  Print
  FullStopFound=True
  Do
    k$=Inkey$
  Loop Until k$<>""
End Sub 'PrintFullStopWarning

Sub bSort(Count)
  '*****************************************************
  'The bubble sort routine to sort the array
  '*****************************************************
  Local n,i
  Flips=1
  Do
    Flips=0
    For n=1 To Count-1
      If a$(n) > a$(n+1) Then
        Swap a$(n),a$(n+1)
        Flips=1
      EndIf
    Next
    'Show progress by printing a dot, unless running under DOS
    If MM.Device$<>"DOS" Then
       i=i+1
       If i Mod 3 = 0 Then Print ".";
    EndIf
  Loop While Flips=1
  Print
End Sub

Sub Swap x$,y$
  '*****************************************************
  ' Swaps two array values
  '*****************************************************
  z$=x$
  x$=y$
  y$=z$
End Sub 'Swap

Sub SelectLowestWord(b$,FileNo)
  '*****************************************************
  'Finds the lowest word from the input file buffers.
  'This is the next one to be written to the CSV file.
  'On entry, b$ contains the highest possible ASCII character.
  'Returns the low word and the file number from which it comes.
  '*****************************************************
  Local i
  For i=1 To NoTempFiles
    If a$(i)<b$ Then
      b$=a$(i)        'Save the lowest word.
      FileNo=i        'Saves the input file number of the lowest word
    EndIf
  Next
End Sub 'SelectLowestWord

Sub WriteTempFile(Count)
  '*****************************************************
  'Write the sorted words to a temp file, dropping any duplicates
  '*****************************************************
  Local PrevWord$,i
  PrevWord$=""
  For i=1 To Count
    If a$(i)<>PrevWord$ Then
      Print #2,a$(i)
      PrevWord$=a$(i)
    Else
      DupCount=DupCount+1
    EndIf
  Next
End Sub ' WriteTempFile

Sub WriteCSVRecord(b$)
  '*****************************************************
  'Write words to a CSV file, packing the data into
  'a number of records of no greater than 240 characters each.
  '*****************************************************
  If Len(CSVRecord$) + Len(b$) > CSVRecLen Then
    Print #9, CSVRecord$
    CSVRecCount=CSVRecCount+1
    CSVRecord$=""
  EndIf
  CSVRecord$ = CSVRecord$ + b$ +","
  RWordCount=RWordCount+1
End Sub 'WriteRWordCSV

Sub WriteTempShortCMDFile(b$)
  '*****************************************************
  'Writes all 1 and 2 character alpha reserved words to a
  'temporary file.
  '*****************************************************
  If Len(b$)<=ShortRWordLen Then
    If Left$(b$,1)>="A" And Left$(b$,1)<="Z" Then
      Print #10, b$
      ShortRWords=ShortRWords+1
    EndIf
  EndIf
End Sub 'WriteTempShortCMDFile

Sub WriteShortCMDFile
  '*****************************************************
  'Copies the short reserved word file, adding the number
  ' of records to the front. This is done so that CRUNCHa
  ' knows how big to create an array of words.
  '*****************************************************
  Open TempShortFile$ For input As #9
  Open ShortCMDFile$ For output As #10
  Print #10, ShortRWords
  Print #10, ShortRWordLen
  Do
    Input #9, a$
    Print #10, a$
  Loop Until Eof(#9)
  Close #9,#10
End Sub 'WriteShortCMDFile

Sub GetFileNames
  '*****************************************************
  ' Input filenames are provided by Command line parameters
  ' Output file names are fixed to support CRUNCH.bas and CRUNCHA.bas
  ' RWSRTMRG infile_1[, infile_2[, infile_3...]]
  ' Delim$ is the parm delimiter.
  '*****************************************************
  Local i,j,Delim$
  If MM.CmdLine$<>"" Then
    ParmString$=MM.CmdLine$
    Print "Getting file names from the command line."
  Else
    Option Error Continue
    Open ParmFile$ For Input As #1
    If MM.ERROR=0 Then
      Line Input #1,ParmString$
      Close #1
      Print "Getting file names from ";ParmFile$
    EndIf
    Option Error Abort
  EndIf

  If ParmString$="" Then
    Print "Couldn't find any input file names in either the command line or in"
    Print ParmFile$;". Please provide them in one or the other."
    End
  EndIf

  ' strip off any trailing blanks or commas from the parm string
  Do While Right$(ParmString$,1)=" " Or Right$(ParmString$,1)=","
      ParmString$ = Left$(ParmString$,Len(ParmString$)-1)
  Loop

  Ptr=1     'p is a pointer to the start of the next file name.
  ' Get input file names and load them into an array
  Dim InFile$(MaxInputFiles)
  NoMoreParms=False
  Print "Input files        : ";
  Do
    GetNextParm(ParmString$,Ptr)
    NbrInputFiles=NbrInputFiles+1
    InFile$(NbrInputFiles)=Parm$
    Print Infile$(NbrInputFiles); "  ";
  Loop Until NoMoreParms=true
  Print:Print "Number input files :";NbrInputFiles
  Print

  'Check Input files exist and ok to write output file
  Option Error Continue
  If testing Then
    Print "Parm string"
    Print ParmString$
    Print "Input file list"
    For i=1 To NbrInputFiles
      Print "!";infile$(i);"!";
    Next
    End
  EndIf
  'If output file exists, ask if ok to overwrite.
  Open CSVFile$ For Input As #9
  If MM.Errno=0 Then
    Close #9
    Input "Output file exists. OK to overwrite (y/n)";Reply$
    If UCase$(Reply$)<>"Y" Then Abort$="y"
  EndIf

  'If short command file exists, ask if ok to overwrite.
  Open ShortCMDFile$ For Input As #10
  If MM.Errno=0 Then
    Close #10
    Input "Short Command file exists. OK to overwrite (y/n)";Reply$
    If UCase$(Reply$)<>"Y" Then Abort$="y"
  EndIf

  'Check each input file exists
  For i=1 To NbrInputFiles
    Open Infile$(i) For Input As #1
    If MM.Errno<>0 Then
      Print infile$;" doesn't exist. Please correct and retry."
      Abort$="y"
    EndIf
    Close #1
  Next
  Option Error Abort

  'If any filename error, quit
  If Abort$="y" Then End
  Print
End Sub

Sub GetNextParm(p$,Strt)
  '**********************************************************
  ' Parse the parameters on the Implied Run Command line,
  ' from CMDparm$ or the DAT file.
  '**********************************************************
  Local p1,p2,A$
  ' Blank and comma are allowed as delimiters
  P1=Instr(Strt,p$," ")
  p2=Instr(Strt,p$,",")
  If p1=0 Then p1=99
  If p2=0 Then p2=99

  If p1=99 And p2=99 Then   'Check for end of parm list
    Parm$=Mid$(p$,Strt)
    NoMoreParms=True
  Else
    If p1>p2 Then p1=p2
    Parm$=Mid$(p$,Strt,P1-Strt)
    Strt=P1
    Do     'Move start pointer to beginning of next file name
      Strt=Strt+1
      A$=Mid$(P$,Strt,1)
    Loop Until (A$<>" " And A$<>",") Or Strt>Len(p$)
  EndIf
End Sub 'GetNextParm
